package us.codecraft.webmagic.processor;

import java.util.ArrayList;
import java.util.List;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.utils.UrlUtils;

/**
 * 非常简单的抽取器。链接抽取使用定义的通配符，并保存抽取整个内容到content字段。<br>
 * @author code4crafter@gmail.com <br>
 * Date: 13-4-22
 * Time: 下午9:15
 */
public class PartOfPageProcessor implements PageProcessor {

    private static final String UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.65 Safari/537.31";
    private Site site;
    private String classEl;

    public PartOfPageProcessor(String startUrl, String classEl) {
        this.site = Site.me().addStartUrl(startUrl).
                setDomain(UrlUtils.getDomain(startUrl)).setUserAgent(UA);

        this.classEl = classEl;

    }

    @Override
    public void process(Page page) {
        List<String> requests = page.getHtml().links().all();
        //调用page.addTargetRequests()方法添加待抓取链接
        page.addTargetRequests(requests);
        //xpath方式抽取
        page.putField("url", page.getUrl());

        for (String s : getTxtByClass(page.getHtml().toString(),classEl)) {
        	page.putField("parts", s);
        }
    }
    
    public List<String> getTxtByClass (String html,String className) {
    	Document doc = Jsoup.parse(html);
  	
    	List<String> li = new ArrayList<String>();
    	Elements elements = doc.select(className);
    	System.out.println(className);
    	System.out.println(elements);
    	for (Element el : elements) {
    		li.add(el.text());
    		System.out.println(el.text());
    	}
    	return li;
    }

    @Override
    public Site getSite() {
        //定义抽取站点的相关参数
        return site;
    }
}
